#!/usr/bin/python

# visualize perhost information, including:
# map tasks per host (success/fail)
# reduce tasks per host (success/fail)
# total file read bytes per host
# total file write bytes per host
# total HDFS read bytes per host
# total shuffle bytes per host
# total map output bytes per host

import sys
import re

def getvalue(a):
	return a[a.find("=")+1:].strip("\"")

def getbytevalue(a):
	return int(a[a.rfind("(")+1:].strip(")"))

def gethostid(a):
	b = int(a[a.find("cloud")+5])
	c = int(a[a.find("cloud")+7:a.rfind("\\")])
	host_per_rack = 50
	return (((b-1)*host_per_rack)+c)

if len(sys.argv) <= 1:
	print "file name?\n"
	sys.exit(-1)

# precess log, get time break down information 
hosts = {}
for l in open(sys.argv[1]):
	A = l.split()
	if A[0] in ["MapAttempt", "ReduceAttempt"]:
	   	type = {}
		status = {}
		id = 0
		counter = {}
		file_read = 0
		file_write = 0
		hdfs_write = 0
		reduce_shuffle = 0
		map_out = 0
		map_in = 0
	   	for a in A:
		   	# read task type
		   	if a.startswith("TASK_TYPE"):
			   	type = getvalue(a)

		   	# read task hostname, translate into host id 
		   	if a.startswith("HOSTNAME"):
			   	id = gethostid(a)

		   	# read task status
		   	if a.startswith("TASK_STATUS"):
			   	status = getvalue(a)
			
			# read counter value
  			if a.startswith("COUNTERS"):
				#counter = (getvalue(a)).strip("}")
				counter = l[l.find("COUNTERS"):]
				B = counter.split(']')
				for b in B:
					if not b == "":
						if b.startswith("COUNTERS"):
							file_read = getbytevalue(b)
						if b.startswith("[(FILE_BYTES_WRITTEN"):
							file_write = getbytevalue(b)
						if b.startswith("[(HDFS"):
							hdfs_read = getbytevalue(b)
						if b.startswith("[(MAP_OUTPUT_BYTES"):
							map_out = getbytevalue(b)
						if b.startswith("[(MAP_INPUT_BYTES"):
							map_in = getbytevalue(b)
						if b.startswith("[(REDUCE_SHUFFLE_BYTES"):
							reduce_shuffle = getbytevalue(b)

	   	# ignore SETUP and CLEANUP task
	   	if type in ["SETUP", "CLEANUP"]:
		   	continue

		# ignore id == 0
		if id == 0:
			continue

		# if no entry for task["hostname"], initialize one
		if not hosts.has_key(id):
			hosts[id] = {}
			hosts[id]["success_map"] = 0
			hosts[id]["fail_map"] = 0
			hosts[id]["success_reduce"] = 0
			hosts[id]["fail_reduce"] = 0
			hosts[id]["file_read"] = 0
			hosts[id]["file_write"] = 0
			hosts[id]["hdfs_read"] = 0
			hosts[id]["reduce_shuffle"] = 0
			hosts[id]["map_out"] = 0
			hosts[id]["map_in"] = 0

	   	if type == "MAP":
	   		if status == "SUCCESS":
				hosts[id]["success_map"] += 1
				hosts[id]["file_read"] += file_read
				hosts[id]["file_write"] += file_write
				hosts[id]["hdfs_read"] += hdfs_read
				hosts[id]["map_out"] += map_out
				hosts[id]["map_in"] += map_in
			else :
				hosts[id]["fail_map"] += 1

	   	if type == "REDUCE":
	   		if status == "SUCCESS":
				hosts[id]["success_reduce"] += 1
				hosts[id]["file_read"] += file_read
				hosts[id]["file_write"] += file_write
				hosts[id]["reduce_shuffle"] += reduce_shuffle
			else :
				hosts[id]["fail_reduce"] += 1


mf = open('tasks.dat', 'w')
for k,v in hosts.items():
	mf.write(str(k)+' '+ str(v["success_map"]) + ' ' + str(v["fail_map"]) + ' ' + str(v["success_reduce"]) + ' ' + str(v["fail_reduce"]) + '\n' )
mf.close()

mf1 = open('maptasks.dat', 'w')
for k,v in hosts.items():
	mf1.write(str(k)+' '+str(v["success_map"])+'\n')
mf1.close()

rf1 = open('reducetasks.dat', 'w')
for k,v in hosts.items():
	rf1.write(str(k)+' '+str(v["success_reduce"])+'\n')
rf1.close()

f1 = open('fileread.dat', 'w')
for k,v in hosts.items():
	f1.write(str(k)+' '+str(v["file_read"])+'\n')
f1.close()


f2 = open('filewrite.dat', 'w')
for k,v in hosts.items():
	f2.write(str(k)+' '+str(v["file_write"])+'\n')
f2.close()


f3 = open('hdfsread.dat', 'w')
for k,v in hosts.items():
	f3.write(str(k)+' '+str(v["hdfs_read"])+'\n')
f3.close()


f4 = open('shuffle.dat', 'w')
for k,v in hosts.items():
	f4.write(str(k)+' '+str(v["reduce_shuffle"])+'\n')
f4.close()


f5 = open('mapout.dat', 'w')
for k,v in hosts.items():
	f5.write(str(k)+' '+str(v["map_out"])+'\n')
f5.close()

f6 = open('mapin.dat', 'w')
for k,v in hosts.items():
	f6.write(str(k)+' '+str(v["map_in"])+'\n')
f6.close()


